package com.sena.crawler;

import com.alibaba.fastjson.JSON;
import com.sena.crawler.domian.Link;
import com.sena.crawler.domian.RootLink;
import com.sena.crawler.enums.LinkTypeEnums;
import com.sena.crawler.loader.PageLoader;
import com.sena.crawler.model.RunConf;
import com.sena.crawler.proxy.ProxyMaker;
import com.sena.crawler.rundata.RunData;
import com.sena.crawler.rundata.strategy.LocalRunData;
import com.sena.crawler.thread.CrawlerThread;

import com.sena.crawler.util.JedisUtilNew;
import com.sena.crawler.util.ReadProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.*;

/**
 * sena crawler
 * <p>
 * seven-05-14 22:44:43
 */
public class SenaCrawler {
    private static Logger logger = LoggerFactory.getLogger(SenaCrawler.class);


    // 分布式的url
    private volatile RunData runData = new LocalRunData();

    // run conf
    private volatile RunConf runConf = new RunConf();

    // thread
    private int threadCount = 1;
    // 爬虫线程数量
    private ExecutorService crawlers = Executors.newCachedThreadPool();
    // 爬虫线程池
    private List<CrawlerThread> crawlerThreads = new CopyOnWriteArrayList<>();

    // ---------------------- get ----------------------

    public RunData getRunData() {
        return runData;
    }

    public RunConf getRunConf() {
        return runConf;
    }


    // ---------------------- builder ----------------------
    public static class Builder {
        private SenaCrawler crawler = new SenaCrawler();

        // run data

        /**
         * 设置运行数据类型
         *
         * @param runData
         * @return Builder
         */
        public Builder setRunData(RunData runData) {
            crawler.runData = runData;
            return this;
        }

        public Builder setSenaCrawlerType(String type) {
            crawler.runConf.setSenaCrawlerType(type);
            return this;
        }

        /**
         * 是否开启正则
         *
         * @param allowRegex
         * @return
         */
        public Builder setAllowRegex(boolean allowRegex) {
            crawler.runConf.setAllowRegex(allowRegex);
            return this;

        }

        /**
         * 待爬的URL列表
         *
         * @param
         * @return Builder
         */
        public Builder setUrls(String... urls) {
            if (urls != null && urls.length > 0) {
                for (String url : urls) {
                    Link link = new Link(url, LinkTypeEnums.ROOT_LINK.getCode(), url, "根网站");
                    crawler.runData.addUrl(link);
                }
            }
            return this;
        }

        public Builder setRootUrls(String url) {
            Link link = new Link(url, LinkTypeEnums.ROOT_LINK.getCode(), url, "根网站");
            crawler.runData.addUrl(link);
            if(!url.contains("?")){
                crawler.getRunConf().setRootLink(url);
            }else {
                crawler.getRunConf().setRootLink(url.substring(0,url.lastIndexOf("?")));
            }
            return this;
        }

        /**
         * json分页
         *
         * @return
         */
        public Builder setPagingUrls() {
            List<String> pagingUrls = ReadProperties.getUrlList("pagingUrl");
            if (pagingUrls != null && pagingUrls.size() > 0) {
                for (String url : pagingUrls) {
                    String[] urls = url.split("@@");
                    Link link = new Link(urls[0], urls[1], LinkTypeEnums.PAGE_LINK.getCode(), urls[2], urls[0], "分页url");
                    //添加到redis中
                    crawler.runData.addUrl(link);
                }
            }
            return this;
        }


        /**
         * 详情
         *
         * @return
         */
        public Builder setDetailUrlsrls() {
            List<String> detailUrls = ReadProperties.getUrlList("detailUrls");
            if (detailUrls != null && detailUrls.size() > 0) {
                try {
                    for (String url : detailUrls) {
                        String[] urls = url.split("@@");
                        Link link = new Link(urls[0], Integer.parseInt(urls[1]), Integer.parseInt(urls[2]));
                        for (int i = link.getStart(); i <= link.getEnd(); i++) {
                            String rlink = link.getLink().replace("##", String.valueOf(i));
                            Link ln = new Link(rlink, LinkTypeEnums.COMMON_LINK.getCode(), rlink, "分页详情");
                            //添加到redis中
                            crawler.getRunData().addUrl(ln);
                        }
                    }
                } catch (Exception e) {
                    logger.info("处理错误");
                }

            }
            return this;
        }


        // run conf

        /**
         * 允许扩散爬取，将会以现有URL为起点扩散爬取整站
         *
         * @param allowSpread
         * @return Builder
         */
        public Builder setAllowSpread(boolean allowSpread) {
            crawler.runConf.setAllowSpread(allowSpread);
            return this;
        }

        /**
         * 关键字匹配
         *
         * @param keys
         * @return
         */
        public Builder setFilterKeyWorld(Set<String> keys) {
            crawler.runConf.setKeyword(keys);
            return this;
        }

        /**
         * 截取后缀
         *
         * @param
         * @return
         */
        public Builder setFilterSuffix(List<String> suffixs) {
            if (suffixs != null && suffixs.size() > 0) {
                Set<String> set = new HashSet<>();
                for (String suffix : suffixs) {
                    set.add(suffix);
                    crawler.getRunConf().setSuffixs(set);
                }
            }
            return this;
        }


        /**
         * 白名单
         *
         * @return
         */
        public Builder setWhiteUrlRegexs() {
            List<String> whiteUrlRegexs = ReadProperties.getUrlList("whiteurl");
            if (whiteUrlRegexs != null) {
                for (String whiteUrlRegex : whiteUrlRegexs) {
                    crawler.runConf.getWhiteUrlRegexs().add(whiteUrlRegex);
                }
            }
            return this;
        }

        /**
         * 黑名单
         *
         * @return
         */
        public Builder setBlackUrlRegexs(List<String> blackurls) {
            if (blackurls != null) {
                for (String blackUrlRegex : blackurls) {
                    crawler.runConf.getBlackUrlRegexs().add(blackUrlRegex);
                }
            }
            return this;
        }

        public Builder setWhiteUrlRegexs(List<String> whiteurls) {
            if (whiteurls != null) {
                for (String whiteUrlRegex : whiteurls) {
                    crawler.runConf.getWhiteUrlRegexs().add(whiteUrlRegex);
                }
            }
            return this;
        }

        /**
         * 黑名单
         *
         * @return
         */
        public Builder setBlackUrlRegexs() {
            List<String> blackUrlRegexs = ReadProperties.getUrlList("blackurl");
            if (blackUrlRegexs != null) {
                for (String blackUrlRegex : blackUrlRegexs) {
                    crawler.runConf.getBlackUrlRegexs().add(blackUrlRegex);
                }
            }
            return this;
        }


        /**
         * 页面下载器
         *
         * @param pageLoader
         * @return Builder
         */
        public Builder setPageLoader(PageLoader pageLoader) {
            crawler.runConf.setPageLoader(pageLoader);
            return this;
        }

        // site

        /**
         * 请求参数
         *
         * @param paramMap
         * @return Builder
         */
        public Builder setParamMap(Map<String, String> paramMap) {
            crawler.runConf.setParamMap(paramMap);
            return this;
        }

        /**
         * 请求Cookie
         *
         * @param cookieMap
         * @return Builder
         */
        public Builder setCookieMap(Map<String, String> cookieMap) {
            crawler.runConf.setCookieMap(cookieMap);
            return this;
        }

        /**
         * 请求Header
         *
         * @param headerMap
         * @return Builder
         */
        public Builder setHeaderMap(Map<String, String> headerMap) {
            crawler.runConf.setHeaderMap(headerMap);
            return this;
        }

        /**
         * 请求UserAgent
         *
         * @param userAgents
         * @return Builder
         */
        public Builder setUserAgent(String... userAgents) {
            if (userAgents != null && userAgents.length > 0) {
                for (String userAgent : userAgents) {
                    if (!crawler.runConf.getUserAgentList().contains(userAgent)) {
                        crawler.runConf.getUserAgentList().add(userAgent);
                    }
                }
            }
            return this;
        }

        /**
         * 请求Referrer
         *
         * @param referrer
         * @return Builder
         */
        public Builder setReferrer(String referrer) {
            crawler.runConf.setReferrer(referrer);
            return this;
        }

        /**
         * 请求方式：true=POST请求、false=GET请求
         *
         * @param ifPost
         * @return Builder
         */
        public Builder setIfPost(boolean ifPost) {
            crawler.runConf.setIfPost(ifPost);
            return this;
        }

        /**
         * 超时时间，毫秒
         *
         * @param timeoutMillis
         * @return Builder
         */
        public Builder setTimeoutMillis(int timeoutMillis) {
            crawler.runConf.setTimeoutMillis(timeoutMillis);
            return this;
        }

        /**
         * 停顿时间，爬虫线程处理完页面之后进行主动停顿，避免过于频繁被拦截；
         *
         * @param pauseMillis
         * @return Builder
         */
        public Builder setPauseMillis(int pauseMillis) {
            crawler.runConf.setPauseMillis(pauseMillis);
            return this;
        }

        /**
         * 代理生成器
         *
         * @param proxyMaker
         * @return Builder
         */
        public Builder setProxyMaker(ProxyMaker proxyMaker) {
            crawler.runConf.setProxyMaker(proxyMaker);
            return this;
        }

        /**
         * 失败重试次数，大于零时生效
         *
         * @param failRetryCount
         * @return Builder
         */
        public Builder setFailRetryCount(int failRetryCount) {
            if (failRetryCount > 0) {
                crawler.runConf.setFailRetryCount(failRetryCount);
            }
            return this;
        }


        /**
         * 失败重试次数，大于零时生效
         *
         * @param maxUrl
         * @return Builder
         */
        public Builder setMixUrlCount(int maxUrl) {
            crawler.runConf.setMaxUrl(maxUrl);
            return this;
        }
        public Builder setNeedMax(boolean b) {
            crawler.runConf.setNeedMax(b);
            return this;
        }

        // thread

        /**
         * 爬虫并发线程数
         *
         * @param threadCount
         * @return Builder
         */
        public Builder setThreadCount(int threadCount) {
            crawler.threadCount = threadCount;
            return this;
        }

        /**
         * Redis天然去重后：设置指定存入的MySQL库表中
         *
         * @param jdbcUrl
         * @return Builder
         */
        public Builder setJdbcUrl(String jdbcUrl) {
            crawler.runConf.setJdbcUrl(jdbcUrl);
            return this;
        }

        public Builder setJedisUtil(JedisUtilNew jedisUtil) {
            crawler.runConf.setJedisUtil(jedisUtil);
            return this;
        }

        /**
         * 设置指定存入的MySQL库表的用户名
         *
         * @param jdbcUserName
         * @return Builder
         */
        public Builder setJdbcUserName(String jdbcUserName) {
            crawler.runConf.setJdbcUserName(jdbcUserName);
            return this;
        }

        /**
         * 设置指定存入的MySQL库表的用户名密码
         *
         * @param jdbcUserPassWord
         * @return Builder
         */
        public Builder setJdbcUserPassWord(String jdbcUserPassWord) {
            crawler.runConf.setJdbcUserPassWord(jdbcUserPassWord);
            return this;
        }

        public SenaCrawler build() {
            return crawler;
        }
    }

    // ---------------------- crawler thread ----------------------

    /**
     * 启动
     *
     * @param sync true=同步方式、false=异步方式
     */
    public void start(boolean sync) {
        if (runData == null) {
            throw new RuntimeException("sena crawler runData can not be null.");
        }
        if (runData.getUrlNum() <= 0) {
            throw new RuntimeException("sena crawler indexUrl can not be empty.");
        }
        if (runConf == null) {
            throw new RuntimeException("sena crawler runConf can not be empty.");
        }
        if (threadCount < 1 || threadCount > 1000) {
            throw new RuntimeException("sena crawler threadCount invalid, threadCount : " + threadCount);
        }
        if (runConf.getPageLoader() == null) {
            throw new RuntimeException("sena crawler pageLoader can not be null.");
        }
        if (runConf.getSenaCrawlerType() == null) {
            throw new RuntimeException("sena crawler pageLoader can not SenaCrawlerType.");
        }
        logger.info(">>>>>>>>>>> sena crawler start ...");


        for (int i = 0; i < threadCount; i++) {
            CrawlerThread crawlerThread = new CrawlerThread(this);
            crawlerThreads.add(crawlerThread);
        }


        for (CrawlerThread crawlerThread : crawlerThreads) {
            crawlers.execute(crawlerThread);
        }
        crawlers.shutdown();
        if (sync) {
            try {
                while (!crawlers.awaitTermination(5, TimeUnit.SECONDS)) {
                    logger.info(">>>>>>>>>>> sena crawler still running ...");
                }
            } catch (InterruptedException e) {
                crawlers.shutdownNow();
                Thread.currentThread().interrupt();
                logger.error(e.getMessage(), e);
            }
        }
    }

    /**
     * 尝试终止
     */
    public void tryFinish() {
        boolean isRunning = false;
        for (CrawlerThread crawlerThread : crawlerThreads) {
            if (crawlerThread.isRunning()) {
                isRunning = true;
                break;
            }
        }
        boolean isEnd = runData.getUrlNum() == 0 && !isRunning;
        if (isEnd) {
            logger.info(">>>>>>>>>>> 爬取完成.");
            stop();
        }

    }

    /**
     * 终止
     */
    public void stop() {
        for (CrawlerThread crawlerThread : crawlerThreads) {
            crawlerThread.toStop();
            logger.info(">>>>>>>>>>> 线程关闭.");

        }
        try {
            setStatus();
            crawlers.shutdownNow();
            logger.info(">>>>>>>>>>> 爬虫停止.");
        } catch (Exception e) {
            logger.info(">>>>>>>>>>> set status is error.");
        }

    }

    public void setStatus() throws IOException {
        try {
            String json = this.getRunConf().getJedisUtil().getStringValue(LinkTypeEnums.ROOT_LINK.getCode());
            RootLink rootLink = JSON.parseObject(json, RootLink.class);
            rootLink.setFinish(true);
            this.getRunConf().getJedisUtil().setStringValue(LinkTypeEnums.ROOT_LINK.getCode(), JSON.toJSONString(rootLink));
        } catch (Exception e) {
            e.printStackTrace();
            logger.info(">>>>>>>>>>> 修改状态失败.");
        } finally {
            this.getRunConf().getJedisUtil().close();

        }

    }


    public void shutdownAndAwaitTermination(ExecutorService pool) {
        if (pool != null && !pool.isShutdown()) {
            pool.shutdown();
            try {
                if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
                    pool.shutdownNow();
                    if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
                        System.out.println("Pool did not terminate");
                    }
                }
            } catch (InterruptedException ie) {
                pool.shutdownNow();
                Thread.currentThread().interrupt();
            }
        }
    }




}
